package FocusCrawler::BestFirst;
our @ISA       = qw(FocusCrawler);
our $VERSION = 1.0;

use warnings;
use strict;
use FocusCrawler qw(&get_new_url &do_leaf_numbering_recursive &collect_features %params %dumps %digests %urls $frontier_db);

use Readonly;
use HTML::TreeBuilder;
use MLDBM qw(DB_File Storable);
use DB_File;
use Storable;
use Lingua::EN::StopWords qw(%StopWords);
use BerkeleyDB;

my $window_width;
my $meta_flag;
our %meta_docs;

sub init {
    my $package = shift;
    $package->SUPER::init(@_);
    $meta_flag = $params{meta_flag};
    $window_width = $params{window_width};
    if ($meta_flag) {
        tie %meta_docs, 'MLDBM', $params{metas_db_file} or die "$!";
    }
}

sub destroy {
    untie %meta_docs if $meta_flag;
}

sub parse_postback {
    my $package = shift;
    my ($content, $url, $base, $label, $h_value, $path, undef) = @_;
    return 0 if $label eq '-';
    my $links_count = 0;
    my $root = HTML::TreeBuilder->new_from_content($content);
    my $index = 0;
    $root->objectify_text();
    do_leaf_numbering_recursive($root, \$index) if $meta_flag;
    $root->attr_encoded(1);
    my @anchors = $root->find_by_tag_name('a');
    foreach my $anchor (@anchors) {
        my $new_url = &get_new_url($anchor, $base);
        next unless defined $new_url;
        if ($meta_flag) {
            my $features_ref = [];
            collect_features($anchor, $root, $features_ref, $index, $window_width) if $meta_flag;
            $meta_docs{$new_url} = $features_ref if $meta_flag;
        }
        my $priority = 1 / (1 + exp(-$h_value));
        $package->add_URL_to_frontier($new_url, $priority, [[$url], $path.'L']);
        $links_count++;
    }
    $root->delete;
    return $links_count;
}

1;
